package crawler.kaiyi;

import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;

import com.zql.entity.AgencyDao;
import com.zql.entity.AgencyEntity2;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/** 
* @author 钟琴隆 E-mail: ai31354907@163.com
* @version 创建时间：2018年10月11日 下午5:15:40 
* 类说明 
*/
public class KaiYiQiChePageProcessor implements PageProcessor {
	private static String pcurl = "http://www.cowinhome.com/data/tms/website_pc/html/scripts/sys_(servicer_){0,1}zone.json";
	@Override
	public void process(Page page) {
		if(page.getUrl().regex(pcurl).match()) {
		String[] provinces = page.getJson().get().split("#");
		List<String> url = new ArrayList<>();
		for(String str : provinces) {
			String[] province = str.split("\\$");
			//如果没有省份和城市,则跳过
			if(province.length<2) {
				continue;
			}
			//提取省和城市
			String proName = province[0];
			String[] cities = province[1].split("\\|");
			for(String cityName : cities) {
				if(page.getUrl().regex("http://www.cowinhome.com/data/tms/website_pc/html/scripts/sys_zone.json").match()) {
					url.add("http://www.cowinhome.com/prj/distributor/dealer?province="+proName+"&city="+cityName+"&pageSize=4&range=50000&pageNum=1");
				}else {
					url.add("http://www.cowinhome.com/prj/servicer/dealer?province="+proName+"&city="+cityName+"&pageSize=4&range=50000&pageNum=1");
				}
			}
			}
			page.addTargetRequests(url);
		}else {
			List<String> name = page.getJson().jsonPath("$.list[*].distributorName").all();
			List<String> address = page.getJson().jsonPath("$.list[*].address").all();
			List<String> serviceTell = page.getJson().jsonPath("$.list[*].salePhone").all();
			List<String> lng = page.getJson().jsonPath("$.list[*].longitude").all();
			List<String> lat = page.getJson().jsonPath("$.list[*].latitude").all();
			List<String> proName = page.getJson().jsonPath("$.list[*].province").all();
			List<String> cityName = page.getJson().jsonPath("$.list[*].city").all();
			List<String> type = new ArrayList<>();
			List<String> dnull = new ArrayList<>();
			String type00 = "";
			if(page.getUrl().get().contains("distributor")) {
				type00 = "经销商";
			}else {
				type00 ="服务商";
			}
			
			for(int i=0;i<name.size();i++) {
				type.add(type00);
				dnull.add("");
			}
				AgencyEntity2 agency = new AgencyEntity2();
				//agency.setdCloseDate(null);
				//agency.setdOpeningDate(null);
				agency.setdUpdateTime(new Timestamp(System.currentTimeMillis()));
				//agency.setnBrandID(-1);
				agency.setsBrand("凯翼");
				//agency.setnDealerIDWeb(-1);
				//agency.setnManufacturerID(-1);
				agency.setsManufacturer("凯翼汽车");
				//agency.setnState(1);
				agency.setsAddress(address);
				agency.setsCity(cityName);
				agency.setsCustomerServiceCall(serviceTell);
				agency.setsDealerName(name);
				agency.setsDealerType(type);
				agency.setsProvince(proName);
				agency.setsSaleCall(dnull);
				agency.setsLongitude(lng);
				agency.setsLatitude(lat);
				//agency.setsControllingShareholder(null);
				//agency.setsOtherShareholders(null);
				new AgencyDao().add(agency);
			//}
		}
	}
	private Site site=Site.me()
							.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
							.setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:63.0) Gecko/20100101 Firefox/63.0")
							.addHeader("Accept-Encoding", "gzip, deflate")
							.addHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3")
							.addHeader("Cache-Control", "max-age=0")
							.addHeader("Connection", "keep-alive")
							.addHeader("Host", "www.cowinhome.com")
							.addHeader("Upgrade-Insecure-Requests", "1")
							//.setCharset("GBK")
			.setSleepTime(0)
			.setTimeOut(60000)
			.setCycleRetryTimes(3)
			//.addHeader("Host","www.cowinhome.com")
			//.addHeader("Cookie", "Hm_lvt_1cce400b006b6e5a7a65449eccdb1ccc=1539249436; Hm_lpvt_1cce400b006b6e5a7a65449eccdb1ccc=1539249484")
			;
	@Override
	public Site getSite() {
		return site;
	}
public static void main(String[] args) {
	System.out.println("爬虫开始：");
	Spider.create(new KaiYiQiChePageProcessor())
	.addUrl("http://www.cowinhome.com/data/tms/website_pc/html/scripts/sys_zone.json")
	.addUrl("http://www.cowinhome.com/data/tms/website_pc/html/scripts/sys_servicer_zone.json")
	.run();
	System.out.println("抓取数据完成！");
}
}
 